{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f0457b74",
   "metadata": {},
   "source": [
    "1.读取数据"
   ]
  },
  {
   "cell_type": "raw",
   "id": "9629a486",
   "metadata": {},
   "source": [
    "#泰坦尼克号幸存顾客预测，通过泰坦尼克号中乘客的数据来预测未来发生船难时哪些人可以幸存，哪些人不能幸存\n",
    "'PassengerId',ID\n",
    "'Survived',是否幸存，0,1\n",
    "'Pclass', 船舱等级\n",
    "'Name',姓名\n",
    "'Sex',性别\n",
    "'Age',年龄\n",
    "'SibSp',带亲属的数量(子女、兄妹)\n",
    "'Parch', 带亲属的数量(父母)\n",
    "'Ticket',船票编号\n",
    "'Fare', 船票价格\n",
    "'Embarked'登录港口"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2f02663b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f01d3bb8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex   Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male  22.0      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  38.0      1   \n",
       "2                             Heikkinen, Miss. Laina  female  26.0      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  35.0      1   \n",
       "4                           Allen, Mr. William Henry    male  35.0      0   \n",
       "\n",
       "   Parch            Ticket     Fare Cabin Embarked  \n",
       "0      0         A/5 21171   7.2500   NaN        S  \n",
       "1      0          PC 17599  71.2833   C85        C  \n",
       "2      0  STON/O2. 3101282   7.9250   NaN        S  \n",
       "3      0            113803  53.1000  C123        S  \n",
       "4      0            373450   8.0500   NaN        S  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('C:/Users/lenovo/BigData/PythonBase/titanic_trains.csv') #路径：绝对路径\n",
    "df = pd.read_csv('./titanic_trains.csv') #相对路径都支持,可以按快捷键tab补全路径\n",
    "df.head() #查看数据的前5行"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f24ebbdf",
   "metadata": {},
   "source": [
    "2缺失值的处理-删除"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "19f9faca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId      0\n",
       "Survived         0\n",
       "Pclass           0\n",
       "Name             0\n",
       "Sex              0\n",
       "Age            177\n",
       "SibSp            0\n",
       "Parch            0\n",
       "Ticket           0\n",
       "Fare             0\n",
       "Cabin          687\n",
       "Embarked         2\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 按列删除(遵循80%法则)如果一列的非缺失部分低于80%则可以考虑删除该列，也就是缺失部分超过20%\n",
    "#axis按列还是行删除 how全部删除还是部分删除 thresh(非缺失值部分数据是多少，891是数据行，0.8是非缺失值占比)，inplace参数表示是否对原始原始\n",
    "#变量进行修改，Ture修改有返回值，false表示不修改并且没有返回值\n",
    "# df_drop = df.dropna(axis='columns',how='any',thresh=891*0.8) #3.10版本不支持how和thresh参数同时出现\n",
    "df_drop = df.dropna(axis='columns',thresh=891*0.8,inplace=False) #axis也可以是1，0，1代表列，0代表行\n",
    "#删掉缺失值占比超过20%的列之后，再看占比情况\n",
    "df_drop.isnull().sum()/df.shape[0]*100\n",
    "df.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a3b650c4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PassengerId    0.0\n",
       "Survived       0.0\n",
       "Pclass         0.0\n",
       "Name           0.0\n",
       "Sex            0.0\n",
       "Age            0.0\n",
       "SibSp          0.0\n",
       "Parch          0.0\n",
       "Ticket         0.0\n",
       "Fare           0.0\n",
       "Embarked       0.0\n",
       "dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#按行删除\n",
    "df_drop_row = df_drop.dropna(axis=0,how='any')#按行删除时不能指定thresh参数\n",
    "df_drop_row.isnull().sum()/891*100  #这样删完以后，整个df_drop_row一个缺失值都没有了"
   ]
  },
  {
   "cell_type": "raw",
   "id": "34949daa",
   "metadata": {},
   "source": [
    "3.数值离散化（特征离散化）：处理的都是数值类型的数据，而且还要求此列数据离散化后有意义，比如本来就是等级的数据，你还做\n",
    "                                离散化，那就没意义了，在做数值离散化之前不能有缺失值，在数据处理时，特征离散化数据处理不做强制要求\n",
    "        离散化效果：\n",
    "            1.会不会买电脑\n",
    "                原始数据Age:1,2,3,4,5,6,7,8,9,10,...100                100个条件\n",
    "                离散化后的数据(其实就是重新划了新的区间范围):幼年:1-12   青年:12-50   中年:50-80    老年:80-100     4个条件\n",
    "        离散化作用:\n",
    "            1.将数据进行区间划分，以减少数据的分布情况，从而提高计算效率，也就是提高模型效率，因为条件变少了嘛\n",
    "            2.可以在一定程度上处理异常值(降低异常值的影响)，怎么理解呢，比如我通过异常值的检测，检测出某个值非常大，比如年龄100岁，如果按异常值的\n",
    "              方法来处理，这个值超过了正常的值，肯定会被计算出为异常值，但是实际生活的情况来说，100岁是正常值，如果进行数值的离散化，那么100就到\n",
    "              老年这个区间了，那么用异常值检测，肯定是属于正常值了，所以可以在一定程度上处理异常值\n",
    "        实现方法：\n",
    "            分箱：常用的方法，下面几种离散化的方式不常用\n",
    "                等宽：将数据划分程N份，每份数据的区间大小相同\n",
    "                      如：[1,2,2,3,4,4,4,5,4,10]分成两份：第一个区间:[1-5]=[1,2,2,3,4,4,4,5,4,]  第二个区间[6-10]=[10]\n",
    "                      缺点：可能出现数据倾斜\n",
    "                等频：将数据划分成N份，每份数据的数据量是相同的\n",
    "                      如：[1,2,2,3,4,4,4,5,4,10]分成两份：第一个区间：5个数据=[1,2,2,3,4]  第二个区间5个数据=[4,4,5,4,10]\n",
    "                      缺点：相同的数出现几种含义，一个数出现几种含义，比如年龄40在几个区间都有，那这个年龄到底是属于青年，中年还是老年\n",
    "            基于聚类的离散化\n",
    "            基于卡方的离散化\n",
    "            基于信息熵的离散化\n",
    "            1R\n",
    "            ..."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dd86b7d",
   "metadata": {},
   "source": [
    "3.1对Age列进行离散化-等宽"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7eda4c76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#pd.cut用来对数据进行离散化 x:离散化对象的数据  bin：区间个数 labels:区间的名字\n",
    "df_drop_row['Age'] = pd.cut(x=df_drop_row['Age'],bins=3,labels=['young','middle','old']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "bbd43ed7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "middle    344\n",
       "young     319\n",
       "old        49\n",
       "Name: Age, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_row['Age'].value_counts()    #value_counts统计列中每个值出现的数量"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca6c600b",
   "metadata": {},
   "source": [
    "3.1对Fare列进行离散化-等频"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0b9cdef6",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_drop_row['Fare'] = pd.qcut(x=df_drop_row['Fare'],q=3,labels=['one','two','three'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d346adb1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "two      241\n",
       "one      239\n",
       "three    232\n",
       "Name: Fare, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_row['Fare'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "53ea1754",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PassengerId</th>\n",
       "      <th>Survived</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Embarked</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Braund, Mr. Owen Harris</td>\n",
       "      <td>male</td>\n",
       "      <td>young</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>one</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Cumings, Mrs. John Bradley (Florence Briggs Th...</td>\n",
       "      <td>female</td>\n",
       "      <td>middle</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>three</td>\n",
       "      <td>C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>Heikkinen, Miss. Laina</td>\n",
       "      <td>female</td>\n",
       "      <td>young</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>one</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Futrelle, Mrs. Jacques Heath (Lily May Peel)</td>\n",
       "      <td>female</td>\n",
       "      <td>middle</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>three</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>Allen, Mr. William Henry</td>\n",
       "      <td>male</td>\n",
       "      <td>middle</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>one</td>\n",
       "      <td>S</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PassengerId  Survived  Pclass  \\\n",
       "0            1         0       3   \n",
       "1            2         1       1   \n",
       "2            3         1       3   \n",
       "3            4         1       1   \n",
       "4            5         0       3   \n",
       "\n",
       "                                                Name     Sex     Age  SibSp  \\\n",
       "0                            Braund, Mr. Owen Harris    male   young      1   \n",
       "1  Cumings, Mrs. John Bradley (Florence Briggs Th...  female  middle      1   \n",
       "2                             Heikkinen, Miss. Laina  female   young      0   \n",
       "3       Futrelle, Mrs. Jacques Heath (Lily May Peel)  female  middle      1   \n",
       "4                           Allen, Mr. William Henry    male  middle      0   \n",
       "\n",
       "   Parch            Ticket   Fare Embarked  \n",
       "0      0         A/5 21171    one        S  \n",
       "1      0          PC 17599  three        C  \n",
       "2      0  STON/O2. 3101282    one        S  \n",
       "3      0            113803  three        S  \n",
       "4      0            373450    one        S  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_drop_row.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
